//
//  Token.cpp
//  LightBasic
//
//  Created by Albert on 23/03/2011.
//  Copyright 2011 LightBasic Development Team. All rights reserved.
//

#include "Token.h"
#include "TokenInfo.h"
#include "StringRef.h"
using namespace lbc;

/// the tokens pool
static boost::pool<> _theTokensPool(sizeof(Token));

/// the id map
typedef boost::unordered_map<std::string, StringRef> TheStringCache;
static TheStringCache _theIdMap;
static std::list<StringRef> _theStringList;

/// Lower memory usage by reusing identical token lexemes and strings
static const StringRef & GetIdentifierEntry(const std::string & str) {
    // longer than 16 bytes
    if (str.length() > 32) {
        *_theStringList.insert(_theStringList.end(), str);
    }
    // cache
    return _theIdMap.insert(
         TheStringCache::value_type(str, str)
    ).first->second;
}


/**
 * token from type only
 */
Token::Token(TokenType type, unsigned int line, unsigned short column, unsigned short length)
    /// TODO shouldn't take length here for the location. Its unrealiable for some tokens!
    : m_type(type), m_lexeme(TokenInfo::GetTokenName(type)), m_location(line, column, length)
{}


/**
 * Create new token
 */
Token::Token(TokenType type, const std::string & lexeme, unsigned int line, unsigned short column, unsigned short length)
    : m_type(type), m_lexeme(GetIdentifierEntry(lexeme)), m_location(line, column, length)
{
}


/**
 * override default operator for pool allocation
 */
void * Token::operator new(size_t)
{
    return _theTokensPool.malloc();
}


/**
 * override default operator for pool deallocate
 */
void Token::operator delete(void * addr)
{
    _theTokensPool.free(addr);
}
